source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
f = file.list()
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
f = list.files()
f
test = fread(f[i])
i = 1
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
test = fread(f[i])[-1:7,]
test = fread(f[i])[-(1:7),]
View(test)
test = fread(f[i])
View(test)
test = fread(f[i], header = F)
test = fread(f[i], header = F)[-(1:7),]
View(test)
temp = fread(f[i], header = F)
View(temp)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
temp[,8]
temp[,8]
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
names(temp) =
temp[,7]
temp[,7]
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
i = 1
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
temp[,7]
View(temp)
temp[8,]
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
temp[8,]
test = temp[8,]
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
names
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(data)
View(test)
View(temp)
d = fread(f[i], header = F)
series = t(d[1,])
ids = t(d[8,])
metadata = data.table(ids = ids, series = series)
d = d[9:nrow(d),]
View(d)
names(d) <- ids
View(d)
d = d[perc == "p99p100"]
View(d)
View(d)
find = c("sfiinc","sfainc", "sptinc")
id.out = NULL
for(i in 1:3){
id.sub = ids[grep(find[i], ids)]
# get rid of male/female only data
male = id.sub[grep("992m", id.sub)]
female = id.sub[grep("992f", id.sub)]
id.sub = id.sub[id.sub!= male & id.sub!= female]
id.out = c(id.sub, id.out)
}
d = gather(d, year)
library(tidy)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
for(i in 1:3){
id.sub = ids[grep(find[i], ids)]
# get rid of male/female only data
male = id.sub[grep("992m", id.sub)]
female = id.sub[grep("992f", id.sub)]
id.sub = id.sub[id.sub!= male & id.sub!= female]
id.out = c(id.sub, id.out)
}
d = gather(d, year)
View(d)
library(data.table)
library(here)
library(tidyr)
dir = here()
wd = paste(dir, "/csv", sep = "")
setwd(wd)
f = list.files()
d = fread(f[i], header = F)
series = t(d[1,])
ids = t(d[8,])
metadata = data.table(ids = ids, series = series)
d = d[9:nrow(d),]
names(d) <- ids
# keep only top1% data
d = d[perc == "p99p100"]
# find series
find = c("sfiinc","sfainc", "sptinc")
id.out = NULL
for(i in 1:3){
id.sub = ids[grep(find[i], ids)]
# get rid of male/female only data
male = id.sub[grep("992m", id.sub)]
female = id.sub[grep("992f", id.sub)]
id.sub = id.sub[id.sub!= male & id.sub!= female]
id.out = c(id.sub, id.out)
}
View(d)
index = 1
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(d)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(d)
View(d)
d = gather(d, year, country)
View(d)
?gather
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(d)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
View(d)
test
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
View(test)
View(d)
test = gather(d, country)
View(test)
test = gather(d, year)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
test
View(d)
melt(d, value.name = c("year", "country"))
melt(d, id.vars =  = c("year", "country"))
melt(d, id.vars =  c("year", "country"))
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(test)
View(test)
View(test)
long = melt(d, id.vars =  c("year", "country"))
names(long) <- c("year", "country", "series", "top1")
long = long[d$series %in% id.out,]
View(long)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
index = 2
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(long)
long$top1 = as.numeric(long$top1)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(final)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(final)
View(long)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(final)
final = na.omit(final)
View(final)
hist(final$top1)
View(final)
View(metadata)
View(metadata)
grep("share", metadata$series.V1)
contains_share =  grep("share", metadata$series.V1)
keep_series = metadata$ids.V1[contains_share]
keep_list = NULL
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
keep_list = unique(keep_list)
final = final[series %in% keep_list]
hist(final$top1)
View(final)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(keep_list)
keep_list
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
View(keep_list)
View(keep_list)
keep_list
test = final[series %in% keep_list$ids.V1]
View(test)
View(keep_list)
hist(test$top1)
hist(test$top1, breaks = 100)
View(keep_list)
keep_list
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
?grepRaw
grep
?grep
f = function(x){grep(x, final$series)}
test = sapply(find, f)
test = sapply(find, f) %>% unlist()
test = sapply(find, f) %>% unlist() %>% unique()
ids = sapply(find, f) %>% unlist() %>% unique()
test = final[ids]
View(test)
male = grep(! "992m", final$series)]
male = grep(! "992m", final$series)
male = grep( "992m", final$series)
# clean
find = c("sfiinc","sfainc", "sptinc")
f = function(x){grep(x, final$series)}
ids = sapply(find, f) %>% unlist() %>% unique()
final = final[ids,]
male = grep( "992m", final$series)
female = grep( "992m", final$series)
female = grep( "992f", final$series)
test = final[!male & !female]
test = final[! c(male, female), ]
View(test)
View(keep_list)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
hist(final$top1)
hist(final$top1, breaks = 20)
hist(final$top1, breaks = 20)
hist(final$top1, breaks = 40)
View(final)
plot(final$year, final$top1)
setwd(dir)
dir
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
system("python filter.py")
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
codes = fread("country codes.csv")
wd
setwd(dir)
codes = fread("country codes.csv")
wd = gsub("Figures", "Empirical Data/World Bank", dir)
setwd(wd)
WB = fread("data_format.csv")
wd = gsub("WID", "World Bank", dir)
setwd(wd)
WB = fread("data_format.csv")
WB = WB[is.na(EG.USE.PCAP.KG.OE) == F]
names(codes)
names(WB)
names(final)
names(final) = c("year", "code2", "series", "top1")
library(data.table)
library(here)
library(tidyr)
system("python filter.py")
dir = here()
wd = paste(dir, "/csv", sep = "")
setwd(wd)
f = list.files()
final = NULL
keep_list = NULL
for (index in 1:length(f) ){
d = fread(f[index], header = F)
series = t(d[1,])
ids = t(d[8,])
metadata = data.table(ids = ids, series = series)
d = d[9:nrow(d),]
names(d) <- ids
# keep only top1% data
d = d[perc == "p99p100"]
# put data into long format
long = melt(d, id.vars =  c("year", "country"))
names(long) <- c("year", "code2", "series", "top1")
long$top1 = as.numeric(long$top1)
final = rbind(final, long)
keep_list = rbind(keep_list, metadata)
}
final = na.omit(final)
# clean
###############################################
find = c("sfiinc","sfainc", "sptinc")
f = function(x){grep(x, final$series)}
ids = sapply(find, f) %>% unlist() %>% unique()
final = final[ids,]
male = grep( "992m", final$series)
female = grep( "992f", final$series)
final = final[! c(male, female), ]
# match with energy and 3 digit codes
############################################
setwd(dir)
codes = fread("country codes.csv")
# energy
wd = gsub("WID", "World Bank", dir)
setwd(wd)
WB = fread("data_format.csv")
WB = WB[is.na(EG.USE.PCAP.KG.OE) == F]
# match column names
names(codes)
names(WB)
names(final)
names(WB) = c("country", "code3", "year", "energy", "gini", "gdp_pc")
names(WB)
names(WB) = c("country", "code3", "year", "energy_pc", "gini", "gdp_pc")
names(codes)
names(WB)
names(final)
?merge
test = merge(final, codes, by = "code2" )
View(test)
merge_1 = merge(final, codes, by = "code2" )
merge_2 = merge(merge_1, WB, by = c("year", "code3"))
names(WB)
final$year = as.numeric(final$year)
# match with energy and 3 digit codes
############################################
setwd(dir)
codes = fread("country codes.csv")
# energy
wd = gsub("WID", "World Bank", dir)
setwd(wd)
WB = fread("data_format.csv")
WB = WB[is.na(EG.USE.PCAP.KG.OE) == F]
# match column names
names(WB) = c("country", "code3", "year", "energy_pc", "gini", "gdp_pc")
names(codes)
names(WB)
names(final)
merge_1 = merge(final, codes, by = "code2" )
merge_2 = merge(merge_1, WB, by = c("year", "code3"))
View(merge_2)
plot(merge_2$gini, merge_2$top1
# export
#######################################3
setwd(dir)
fwrite(final, "top_1.csv")
plot(merge_2$gini, merge_2$top1)
plot(merge_2$energy_pc, merge_2$top1)
plot(merge_2$energy_pc, merge_2$top1, log = "x")
View(merge_2)
plot(merge_2$gini, merge_2$top1)
View(final)
conf = function(x){
quantile(x, probs = c(0.025, 0.5, 0.975))
}
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
dir = here()
dir
dir = here()
setwd(dir)
system("python filter.py")
dir = here()
setwd(dir)
system("python filter.py")
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/gini.R')
View(final)
library(data.table)
library(here)
library(tidyr)
dir = here()
setwd(dir)
system("python filter.py")
wd = paste(dir, "/csv", sep = "")
setwd(wd)
f = list.files()
final = NULL
keep_list = NULL
index  = 1
d = fread(f[index], header = F)
series = t(d[1,])
data_only = d[9:nrow(d),]
View(data_only)
names(data_only) <- ids
series = t(d[1,])
ids = t(d[8,])
data_only = d[9:nrow(d),]
names(data_only) <- ids
long = melt(d_only, id.vars =  c("year", "country"))
long = melt(data_only, id.vars =  c("year", "country"))
View(long)
View(data_only)
long = melt(data_only, id.vars =  c("year", "country", "perc"))
View(long)
long = melt(data_only, id.vars =  c("year", "country", "perc"))
names(long) <- c("year", "code2", "perc", "series", "value")
long$value = as.numeric(long$value)
final = rbind(final, long)
d = fread(f[index], header = F)
# meta data
series = t(d[1,])
ids = t(d[8,])
# data only
data_only = d[9:nrow(d),]
names(data_only) <- ids
# put data into long format
long = melt(data_only, id.vars =  c("year", "country", "perc"))
names(long) <- c("year", "code2", "perc", "series", "value")
long$value = as.numeric(long$value)
final = rbind(final, long)
library(data.table)
library(here)
library(tidyr)
dir = here()
setwd(dir)
system("python filter.py")
wd = paste(dir, "/csv", sep = "")
setwd(wd)
f = list.files()
final = NULL
for (index in 1:length(f) ){
d = fread(f[index], header = F)
# meta data
series = t(d[1,])
ids = t(d[8,])
# data only
data_only = d[9:nrow(d),]
names(data_only) <- ids
# put data into long format
long = melt(data_only, id.vars =  c("year", "country", "perc"))
names(long) <- c("year", "code2", "perc", "series", "value")
long$value = as.numeric(long$value)
final = rbind(final, long)
}
final = na.omit(final)
print(index/length(f))
print( round( index/length(f )) )
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/gini.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/gini.R')
pb <- tkProgressBar(title = "progress bar", min = 0, max = length(f), width = 300)
library(tcltk)
pb <- tkProgressBar(title = "progress bar", min = 0, max = length(f), width = 300)
setTkProgressBar(pb, index, label=paste( round(i/length(f)*100, 0))
}
final = na.omit(final)
# clean
###############################################
find = c("sfiinc","sfainc", "sptinc")
f = function(x){grep(x, final$series)}
ids = sapply(find, f) %>% unlist() %>% unique()
final = final[ids,]
male = grep( "992m", final$series)
female = grep( "992f", final$series)
final = final[! c(male, female), ]
final$year = as.numeric(final$year)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/gini.R')
pb <- txtProgressBar(min = 0, max = total, style = 3)
pb <- txtProgressBar(min = 0, max = length(f), style = 3)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/gini.R')
View(final)
setwd(dir)
View(final)
setwd(dir)
fwrite(final, "all_data.csv")
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format_top_1.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/WID/format_top_1.R')
warnings()
setwd(dir)
library(data.table)
library(here)
library(tidyr)
library(tcltk)
dir = here()
setwd(dir)
system("python filter.py")
wd = paste(dir, "/csv", sep = "")
setwd(wd)
f = list.files()
final = NULL
pb <- txtProgressBar(min = 0, max = length(f), style = 3)
for (index in 1:length(f) ){
d = fread(f[index], header = F)
# meta data
series = t(d[1,])
ids = t(d[8,])
# data only
data_only = d[9:nrow(d),]
names(data_only) <- ids
# put data into long format
long = melt(data_only, id.vars =  c("year", "country", "perc"))
names(long) <- c("year", "code2", "perc", "series", "value")
long$value = as.numeric(long$value)
final = rbind(final, long)
# progress bar
setTxtProgressBar(pb, index)
}
final = na.omit(final)
# clean
###############################################
find = c("sfiinc","sfainc", "sptinc")
f = function(x){grep(x, final$series)}
ids = sapply(find, f) %>% unlist() %>% unique()
final = final[ids,]
male = grep( "992m", final$series)
female = grep( "992f", final$series)
final = final[! c(male, female), ]
final$year = as.numeric(final$year)
# merge with codes
#########################################
setwd(dir)
codes = fread("country codes.csv")
merge = merge(final, codes, by = "code2" )
View(merge)
